package com.yangyang.ralearn.client;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Random;
import java.util.Set;

import org.apache.log4j.Logger;
import org.springframework.stereotype.Component;

import com.yangyang.ralearn.common.BaseAlgorithm;
import com.yangyang.ralearn.common.Constants;
import com.yangyang.ralearn.common.JEnvironment;
import com.yangyang.ralearn.common.entity.base.AbstractDomainPredicate;
import com.yangyang.ralearn.common.entity.base.ActionLimit;
import com.yangyang.ralearn.common.entity.base.BaseAction;
import com.yangyang.ralearn.common.entity.base.BaseActionModelDef;
import com.yangyang.ralearn.common.exception.AlgorithmEndException;
import com.yangyang.ralearn.common.exception.ErrorActionException;
import com.yangyang.ralearn.common.exception.ReachGoalException;
import com.yangyang.ralearn.common.types.JType;

@Component
public class Ralearn extends BaseAlgorithm implements
		IClientEnvironmentListener {

	private Random random = new Random();
	protected JEnvironment environment;
	protected List<AbstractDomainPredicate> goalStates;

	/*
	 * (non-Javadoc)
	 * 
	 * @see
	 * com.yangyang.ralearn.client.IClientEnvironmentListener#init(com.yangyang
	 * .ralearn.common.JEnvironment, java.util.List, java.util.Map)
	 */
	@Override
	public void init(JEnvironment environment,
			List<AbstractDomainPredicate> goalStates) {
		this.environment = environment;
		this.goalStates = goalStates;
		Map<String, Map<String, String>> actions = new HashMap<String, Map<String, String>>();
		{
			for (ActionLimit limit : environment.mapName2ActionLimit.values()) {
				Map<String, String> params = new HashMap<String, String>();
				for (Entry<String, JType> entry : limit.getMapVar2Type()
						.entrySet()) {
					params.put(entry.getKey(), entry.getValue().getName());
				}
				actions.put(limit.getName(), params);
			}
		}
		super.init(environment, actions);
	}

	@Override
	public void run() throws AlgorithmEndException {
		// TODO Auto-generated method stub
		Logger.getLogger(Ralearn.class).debug(this.mapName2Models);

		int counter = 0;
		List<String> actionSignature = new ArrayList<String>();
		actionSignature.addAll(this.mapName2Models.keySet());
		List<AbstractDomainPredicate> currentStates = ((RLClientHandler) RLClientHandler
				.instance()).observe();
		Logger.getLogger(Ralearn.class).debug(
				"***************************************");
		while (counter++ < Constants.MAX_ITER_TIME) {

			// reach the goal
			if (currentStates.containsAll(goalStates)) {
				Logger.getLogger(Ralearn.class).debug(
						"***************************************");

				Logger.getLogger(Ralearn.class).error(this.mapName2Models);
				throw new AlgorithmEndException("reach the goal!");
			}

			// randomly choose an action a
			String actionName = actionSignature.get(random
					.nextInt(actionSignature.size()));

			//
			BaseActionModelDef model = this.getActionModel(actionName);
			// these actions are all actions that can do in currentStates witch
			// action-model: model
			Set<BaseAction> actions = this.getAllPossiableAction(model,
					currentStates);
			boolean removeFlag = false;
			for (BaseAction action : actions)
				if (BaseActionModelDef.canDo(action, model, currentStates)) {
					if (!((RLClientHandler) RLClientHandler.instance())
							.canDo(action.toString())) {
						this.mapName2Models.get(actionName).remove(model);
						removeFlag = true;
						Logger.getLogger(Ralearn.class).debug(model);
						break;
					}
				}
			if (!removeFlag) {
				try {

					List<BaseAction> list = new ArrayList<BaseAction>();
					list.addAll(actions);
					BaseAction action = list.get(random.nextInt(list.size()));
					((RLClientHandler) RLClientHandler.instance())
							.doAction(action.toString());
					currentStates = ((RLClientHandler) RLClientHandler
							.instance()).observe();
					int qValue = this.getReward(
							BaseActionModelDef.doAction(action, model),
							currentStates);
					if (qValue == -Constants.MAX_SCORE) {
						this.mapName2Models.get(actionName).remove(model);
						Logger.getLogger(Ralearn.class).debug(model);
					} else {
						model.setQValue((int) (qValue * Constants.alpha + (1 - Constants.alpha)
								* model.getQValue()));
					}
				} catch (ReachGoalException e) {
					// TODO Auto-generated catch block
					e.printStackTrace();

				} catch (ErrorActionException e) {
					// TODO Auto-generated catch block
					// e.printStackTrace();
				}
			}

		}
		Logger.getLogger(Ralearn.class).debug(
				"***************************************");
		Logger.getLogger(Ralearn.class).error(this.mapName2Models);

	}

	/**
	 * 获取当前动作下Q值最大的动作模型
	 * 
	 * @param action
	 * @return
	 */
	public BaseActionModelDef getActionModel(String action) {
		List<BaseActionModelDef> set = new ArrayList<BaseActionModelDef>();
		int maxQValue = -1;
		for (BaseActionModelDef def : this.mapName2Models.get(action)) {
			if (def.getQValue() > maxQValue) {
				set.clear();
				set.add(def);
				maxQValue = def.getQValue();
			} else if (def.getQValue() == maxQValue) {
				set.add(def);
			}
		}
		return set.get(random.nextInt(set.size()));
	}

	/**
	 * 根据预期状态和实际状态获取相应的奖励
	 * 
	 * @param expectedPredicates
	 * @param observedPredicates
	 * @return
	 */
	public int getReward(List<AbstractDomainPredicate> expectedPredicates,
			List<AbstractDomainPredicate> observedPredicates) {
		int reward = 0;
		AbstractDomainPredicate[] predicates = observedPredicates
				.toArray(new AbstractDomainPredicate[0]);
		for (AbstractDomainPredicate predicate : expectedPredicates) {
			if (predicate.cntraPredicate(predicates))
				return -Constants.MAX_SCORE;
		}
		for (AbstractDomainPredicate predicate : expectedPredicates) {
			if (expectedPredicates.contains(predicate))
				reward += 100;
		}
		return reward;
	}

	@Override
	public String getName() {
		// TODO Auto-generated method stub
		return "Ralearn 1.0";
	}

	public static void main(String[] args) {

		Ralearn algorithm = new Ralearn();
		RLClientHandler.instance().start();
		((RLClientHandler) RLClientHandler.instance()).init("bin/domain1.pddl",
				"bin/problem1.pddl", algorithm);

		System.out.println("client is running ...");

		try {
			algorithm.filter();
			algorithm.run();
			Thread.sleep(5000);
		} catch (InterruptedException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (AlgorithmEndException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		System.out.println("client finished!");
	}

	@Override
	public void filter() {
		// TODO Auto-generated method stub
		List<String> actionSignature = new ArrayList<String>();
		actionSignature.addAll(this.mapName2Models.keySet());
		// random select an action a
		if (true) {
			List<AbstractDomainPredicate> currentStates = ((RLClientHandler) RLClientHandler
					.instance()).observe();
			String actionName = actionSignature.get(random
					.nextInt(actionSignature.size()));
			Set<BaseActionModelDef> toRemoveModels = new HashSet<BaseActionModelDef>();
			for (BaseActionModelDef model : this.mapName2Models.get(actionName)) {
				Logger.getLogger(Ralearn.class.getName()).debug(model);
				Set<BaseAction> actions = this.getAllPossiableAction(model,
						currentStates);
				for (BaseAction action : actions) {
					if (BaseActionModelDef.canDo(action, model, currentStates)
							&& !((RLClientHandler) RLClientHandler.instance())
									.canDo(action.toString()))
						toRemoveModels.add(model);
					break;
				}
			}
			Logger.getLogger(Ralearn.class).debug(
					"-------------------------------------------");
			Logger.getLogger(Ralearn.class).debug(
					this.mapName2Models.get("stack").size());
			this.mapName2Models.get(actionName).removeAll(toRemoveModels);
			Logger.getLogger(Ralearn.class).debug(
					this.mapName2Models.get("stack").size());
			// Logger.getLogger(Ralearn.class).debug(toRemoveModels);
			Logger.getLogger(Ralearn.class).debug(
					"-------------------------------------------");

		}

		// Logger.getLogger(Ralearn.class).debug(this.mapName2Models);

	}

}
